{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = '/home/husein/t5/prepare/mesolitica-tpu.json'\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from pegasus import transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = 32000\n",
    "hidden_size = 768\n",
    "filter_size = 3072\n",
    "num_encoder_layers = 12\n",
    "num_decoder_layers = 12\n",
    "num_heads = 12\n",
    "label_smoothing = 0.0\n",
    "dropout = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/malaya/Malaya/pretrained-model/pegasus/pegasus/layers/attention.py:46: The name tf.layers.Dense is deprecated. Please use tf.compat.v1.layers.Dense instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = transformer.TransformerEncoderDecoderModel(vocab_size, hidden_size,\n",
    "                                                   filter_size, num_heads,\n",
    "                                                   num_encoder_layers,\n",
    "                                                   num_decoder_layers,\n",
    "                                                   label_smoothing, dropout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/malaya/Malaya/pretrained-model/pegasus/pegasus/layers/embedding.py:61: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/malaya/Malaya/pretrained-model/pegasus/pegasus/layers/embedding.py:61: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/malaya/Malaya/pretrained-model/pegasus/pegasus/layers/embedding.py:65: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/malaya/Malaya/pretrained-model/pegasus/pegasus/layers/embedding.py:69: calling RandomNormal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/husein/malaya/Malaya/pretrained-model/pegasus/pegasus/layers/attention.py:131: The name tf.matrix_band_part is deprecated. Please use tf.linalg.band_part instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/malaya/Malaya/pretrained-model/pegasus/pegasus/layers/decoding.py:96: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/malaya/Malaya/pretrained-model/pegasus/pegasus/layers/decoding.py:97: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "X = tf.placeholder(tf.int64, (None, None))\n",
    "top_p = tf.placeholder(tf.float32, None, name = 'top_p')\n",
    "temperature = tf.placeholder(tf.float32, None, name = 'temperature')\n",
    "outputs = model.predict({\"inputs\": X,}, tf.shape(X)[1], beam_size = 1,\n",
    "                       top_p = top_p, temperature = temperature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'logits:0' shape=(?, ?) dtype=int64>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = tf.identity(outputs['outputs'], name = 'logits')\n",
    "logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "\n",
    "vocab = '/home/husein/b2b/sp10m.cased.t5.model'\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(vocab)\n",
    "\n",
    "class Encoder:\n",
    "    def __init__(self, sp):\n",
    "        self.sp = sp\n",
    "        self.vocab_size = sp.GetPieceSize() + 100\n",
    "    \n",
    "    def encode(self, s):\n",
    "        return self.sp.EncodeAsIds(s)\n",
    "    \n",
    "    def decode(self, ids, strip_extraneous=False):\n",
    "        return self.sp.DecodeIds(list(ids))\n",
    "    \n",
    "encoder = Encoder(sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'gs://mesolitica-tpu-general/pegasus-summarization-base/model.ckpt-900000'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "ckpt_path = tf.train.latest_checkpoint('gs://mesolitica-tpu-general/pegasus-summarization-base')\n",
    "ckpt_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from gs://mesolitica-tpu-general/pegasus-summarization-base/model.ckpt-900000\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver()\n",
    "saver.restore(sess, ckpt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "def cleaning(string):\n",
    "    return re.sub(r'[ ]+', ' ', unidecode(string.replace('\\n', ' '))).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = \"\"\"\n",
    "Tidak ada apa-apa mengenai mesyuarat fakulti biologi Jumaat lalu di University of Alabama di Huntsville yang membayangkan pembunuhan yang akan berlaku, Profesor Debra Moriarity memberitahu wartawan Rabu. \"Ia sebenarnya adalah satu-satunya mesyuarat fakulti yang sangat santai dan biasa,\" ahli biokim memberitahu CNN affiliate WAAY mengenai 13 orang yang duduk di sekitar meja bujur di Bilik 369 di Shelby Centre for Science and Technology. \"Peristiwa akan datang, kelas penjadualan, belanjawan. Ia sebenarnya merupakan salah satu daripada mesyuarat fakulti yang paling mudah yang kami ada.\" Antara peserta adalah Amy Bishop, seorang ahli genetik yang terlatih Harvard dengan siapa Moriarity telah membangun hubungan profesional yang dipupuk oleh hakikat bahawa kedua-dua wanita bekerja dengan budaya sel. \"Kadang-kadang anda meminjam sesuatu dari satu sama lain,\" katanya. \"Kami telah bercakap mengenai cadangan pemberian cadangan bersama.\" Selepas kira-kira sejam, sebelum 4hb, Bishop - yang baru-baru ini dinafikan tempohnya - menamatkan tenang. \"Secara tiba-tiba, dia hanya berdiri dan ditembak,\" kata Moriarity. Moriarity bertindak balas dengan cepat, menjatuhkan ke tangan dan lututnya di atas permaidani kelabu. \"Hanya jatuh ke lantai dan merangkak di bawah meja dan merangkak ke arah Amy,\" kata Moriarity, yang memberi tumpuan kepada satu perkara. \"Maksud saya, kamu merangkak di bawah meja, kamu melihat kaki orang yang menembak di atas meja, saya merangkul kakinya dan, saya tidak tahu apa yang saya fikirkan, saya tidak memikirkan apa-apa. hanya berfikir: \\'Ambil dia!\\' \"Dan dia menghalang saya. Maksud saya, dia menarik kaki kakinya percuma dan saya berada di ambang pintu dengan belakang saya jenisnya. Dan saya fikir dia cuba menembak saya, tetapi ketika saya mula berteriak kepadanya, \\'Amy, Amy, berfikir tentang cucu saya, berfikir tentang anak perempuan saya! Ini saya! Saya telah membantu anda sebelum ini; Saya akan membantu anda lagi! Jangan buat Amy ini! Jangan lakukan ini! \\'\"Uskup kemudian melangkah masuk ke dalam dewan, menunjuk pistol di Moriarity dan menarik pencetus, kata ahli biologi.\" Ia diklik, dan ia diklik lagi, dan saya merangkak kembali ke dalam bilik dan menutup pintu dan dia ditinggalkan di dalam dewan. \"Orang-orang yang terselamat itu beraksi. Satu orang mengunci pintu kayu, satu lagi mengetuk meja ke atasnya, yang lain memindahkan peti sejuk ke tempatnya untuk menghalang pintu, yang lain dipanggil 911, yang lain berpindah ke Tiga orang yang maut, tiga lagi yang cedera, dua daripadanya kekal dirawat di rumah sakit pada hari Rabu dalam keadaan kritikal, menurut seorang jurubicara Hospital Huntsville, yang ketiga dilepaskan, Moriarity, yang menyertai fakulti sekolah pada tahun 1984, mengatakan bahawa mangsa tidak menjejaskan rancangannya untuk kekal di sekolah dan dia menolak apa-apa cadangan bahawa peranannya untuk mendapatkan Bishop di luar bilik itu adalah heroik. \"Dia mengikuti saya di dewan dan kemudian senjata itu macet dan saya boleh mendapatkan ba ck di dalam bilik, \"kata Moriarity. \"Itu bukan seorang pahlawan. Itu hanya Tuhan yang melihat kamu.\" Dia berkata dia mempunyai sedikit masa untuk berfikir. \"Dari awal hingga akhirnya kami mendapat sesuatu yang tidak dapat dibendung, ia tidak boleh melebihi 20 saat,\" katanya. Moriarity terus menolak cadangan bahawa apa-apa boleh dilakukan untuk melindungi mangsa. \"Tidak ada cara untuk menjangka ini,\" katanya. \"Dan tiada apa yang dapat dilakukan untuk menghentikannya, semuanya berlaku terlalu pantas.\" Dan dia bimbang bahawa apa-apa cubaan untuk mengetatkan keselamatan boleh membawa kesan negatif. \"Ada kejahatan di dunia, malangnya orang baik disakiti oleh itu, tetapi universiti adalah tempat pemikiran bebas dan kebebasan untuk meneroka idea-idea dan mencari pengetahuan baru dan anda tidak mahu meletakkan sesuatu di tempat yang meredakannya. \" Moriarity kembali ke pejabatnya pada hari Rabu dan berkata dia merancang untuk meneruskan pengajaran minggu depan. Dia meramalkan bahawa, dengan bantuan ubat anti-kecemasan, dia akan dapat tidur malam Rabu. \"Saya telah bercakap dengan keluarga dan kawan-kawan dan hanya mendapatkan sokongan mereka membantu anda menanganinya,\" katanya. \"Saya fikir sekarang kebanyakan dari kita mahu kembali ke sana dan mendapatkan sesuatu yang berlaku, membuat rancangan untuk siapa yang akan menutup kelas.\" Satu perkhidmatan peringatan untuk menghormati kehidupan orang mati - ahli fakulti Maria Davis, Adriel Johnson dan Gopi Podila - akan diadakan pada hari Jumaat.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "string2 = \"\"\"\n",
    "Gabungan parti Warisan, Pakatan Harapan, dan Upko hari ini mendedahkan calon-calon masing-masing untuk pilihan raya negeri Sabah, tetapi ketika pengumuman itu berlangsung, perwakilan PKR di dewan itu dilihat ‘gelisah’ seperti ‘tidak senang duduk’.\n",
    "\n",
    "Sekumpulan anggota PKR kemudian dilihat meninggalkan dewan di Pusat Konvensyen Antarabangsa Sabah di Kota Kinabalu selepas berbincang dengan ketua PKR Sabah Christina Liew.\n",
    "\n",
    "Semakan senarai-senarai calon berkenaan mendapati PKR hanya memperolehi separuh daripada jumlah kerusi yang diharapkan.\n",
    "\n",
    "Semalam, PKR Sabah mengumumkan akan bertanding di 14 kerusi tetapi ketika Presiden Warisan Shafie Apdal mengumumkan calon gabungan tersebut hari ini, PKR hanya diberikan tujuh kerusi untuk bertanding.\n",
    "\n",
    "Kerusi yang diberikan adalah Api-Api, Inanam, Tempasuk, Tamparuli, Matunggong, Klias, dan Sook.\n",
    "\n",
    "Klias dan Sook adalah dua kerusi yang diberikan kepada PKR, sementara lima kerusi selebihnya pernah ditandingi oleh PKR pada pilihan raya umum 2018.\n",
    "\n",
    "Dalam pengumuman PKR Sabah semalam, parti itu menjangkakan Warisan akan turut menyerahkan kerusi Kemabong, Membakut, dan Petagas kepada mereka.\n",
    "\n",
    "Walau bagaimanapun, Warisan menyerahkan kerusi Kemabong kepada Upko dan mengekalkan bertanding untuk kerusi Membakut dan Petagas.\n",
    "\n",
    "PKR juga menuntut empat daripada 13 kerusi baru yang diperkenalkan iaitu Segama, Limbahau, Sungai Manila, dan Pintasan tetapi Warisan membolot semua kerusi itu.\n",
    "\n",
    "Sebagai pertukaran untuk kerusi yang diintainya, PKR bersedia untuk menyerahkan kerusi Kadaimaian, Kuala Penyu, dan Karanaan. Namun, ini dijangka tidak akan berlaku memandangkan parti tersebut tidak berpuas hati dengan agihan kerusi seperti yang diharapkan itu.\n",
    "\n",
    "Selepas perwakilan dari PKR dan Liew keluar dari dewan tersebut, wartawan kemudian menyusuri Liew untuk mendapatkan penjelasannya.\n",
    "\n",
    "Walau bagaimanapun, Liew enggan memberikan sebarang komen dan berkata bahawa dia ingin ke tandas.\n",
    "\n",
    "Liew dan perwakilan PKR kemudian tidak kembali ke dalam dewan tersebut.\n",
    "\n",
    "Apabila calon pilihan raya yang diumumkan diminta naik ke atas pentas untuk sesi bergambar, Liew tidak kelihatan.\n",
    "\n",
    "Bilangan kerusi yang ditandingi oleh PKR kali ini hanya kurang satu kerusi daripada yang ditandingi parti itu pada PRU 2018.\n",
    "\n",
    "Dalam perkembangan berkaitan, DAP dan Amanah dikatakan tidak mempunyai sebarang masalah dengan kerusi yang diberikan untuk PRN Sabah.\n",
    "\n",
    "Sementara itu, Presiden Upko Madius Tangau enggan mengulas adakah dia berpuas hati dengan agihan kerusi tersebut. Madius kekal di majlis tersebut sehingga ia berakhir.\n",
    "\n",
    "Partinya diberikan 12 kerusi, iaitu lebih tujuh kerusi berbanding PRU lalu.\n",
    "\n",
    "DAP dan Amanah akan bertanding di bawah logo Warisan sementara PKR dan Upko akan menggunakan logo masing-masing.\n",
    "\n",
    "DAP akan bertanding di tujuh kerusi, jumlah yang sama seperti yang mereka tandingi pada PRU lalu, sementara Amanah diberi satu kerusi.\n",
    "\n",
    "Warisan akan bertanding sebanyak 54 kerusi.\n",
    "\n",
    "Perkembangan terbaru ini mungkin mencetuskan pergeseran di antara PKR dan Warisan. PKR boleh memilih untuk bertanding di lebih banyak kerusi daripada 14 yang dituntutnya manakala Warisan juga boleh bertanding di kerusi sekutunya.\n",
    "\n",
    "Barisan pemimpin tertinggi PKR dan Warisan hanya mempunyai dua hari sebelum hari penamaan calon pada Sabtu untuk mengurangkan pergeseran.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "string3 = \"\"\"\n",
    "Penubuhan universiti sukan seperti diutarakan Ketua Unit Sukan Kementerian Pengajian Tinggi, Dr Pekan Ramli dan disokong Pakar Pembangunan Sukan dan Reakreasi Luar, Universiti Pendidikan Sultan Idris (UPSI), Prof Dr Md Amin Md Taaf seperti disiarkan akhbar ini, memberikan sinar harapan kepada kewujudan institusi sedemikian.\n",
    "\n",
    "Ia menjadi impian atlet negara untuk mengejar kejayaan dalam bidang sukan dan kecemerlangan dalam akademik untuk menjamin masa depan lebih baik apabila bersara daripada arena sukan kelak.\n",
    "\n",
    "Pelbagai pandangan, idea, kaedah, bukti dan cadangan dilontarkan pakar berikutan pentingnya universiti sukan yang akan memberi impak besar sama ada pada peringkat kebangsaan mahupun antarabangsa.\n",
    "\n",
    "Negara lain sudah lama meraih laba dengan kewujudan universiti sukan seperti China, Korea, Japan, Taiwan, India dan Vietnam. Mereka menghasilkan atlet universiti yang mempamerkan keputusan cemerlang pada peringkat tinggi seperti Sukan Olimpik, Kejohanan Dunia dan Sukan Asia.\n",
    "\n",
    "Justeru, kejayaan mereka perlu dijadikan rujukan demi memajukan sukan tanah air. Jika kita merujuk pendekatan Asia, kewujudan universiti sukan penting dan memberi kesan positif dalam melonjakkan prestasi sukan lebih optimum.\n",
    "\n",
    "Namun, jika kita melihat pendekatan Eropah, universiti sukan bukan antara organisasi atau institusi penting yang diberi perhatian dalam menyumbang kepada pemenang pingat.\n",
    "\n",
    "Antara isu dalam universiti sukan ialah kos tinggi, lokasi, prasarana sukan, pertindihan kursus dengan universiti sedia ada dan impak terhadap dunia sukan negara hingga mengundang persoalan kewajaran dan kerelevanan penubuhannya.\n",
    "\n",
    "Namun sebagai bekas atlet memanah negara dan Olympian (OLY) di Sukan Olimpik 2004 di Athens, Greece serta bekas pelajar Sekolah Sukan Bukit Jalil hingga berjaya dalam dunia akademik, saya mendapati terdapat beberapa faktor sering menjadi halangan dalam rutin harian mereka.\n",
    "\n",
    "Antaranya, faktor masa yang terpaksa bergegas menghadiri kuliah selepas tamat sesi latihan yang mengambil masa 15 hingga 20 minit dengan menunggang motosikal; kereta (20-30 minit) atau pengangkutan disediakan Majlis Sukan Negara (MSN) ke Universiti Putra Malaysia (UPM).\n",
    "\n",
    "Jika mereka menuntut di Universiti Teknologi MARA (UiTM) atau Universiti Malaya (UM), ia mungkin lebih lama.\n",
    "\n",
    "Walaupun di universiti tersedia dengan kemudahan kolej dan kemudahan sukan, mereka memilih pulang ke MSN untuk menjalani latihan bersama pasukan dan jurulatih di padang atau gelanggang latihan rasmi.\n",
    "\n",
    "Ini berlanjutan selagi bergelar atlet negara yang perlu memastikan prestasi sentiasa meningkat dari semasa ke semasa tanpa mengabaikan tugas sebagai pelajar.\n",
    "\n",
    "Alangkah baiknya jika sebahagian Sekolah Sukan Bukit Jalil itu sendiri dijadikan Kolej atau Universiti Sukan Malaysia kerana lengkap dari segi kemudahan prasarana sukannya dan proses pengajaran dan pembelajaran (PdP) dalam bidang Sains Sukan, Kejurulatihan, Pendidikan Jasmani dan setaraf dengannya.\n",
    "\n",
    "Pengambilan setiap semester pula hanya terhad kepada atlet berstatus kebangsaan dan antarabangsa sahaja supaya hasrat melahirkan lebih ramai atlet bertaraf Olimpik mudah direalisasikan.\n",
    "\n",
    "Contohnya, bekas atlet lompat bergalah negara, Roslinda Samsu yang juga pemenang pingat perak Sukan Asia Doha 2006 dan Penerima Anugerah Khas Majlis Anugerah Sukan KPT 2012, terpaksa mengambil masa lebih kurang sembilan tahun untuk menamatkan ijazah Sarjana Muda Pendidikan Jasmani di UPM sepanjang 14 tahun terbabit dalam sukan olahraga.\n",
    "\n",
    "Sepanjang tempoh bergelar atlet kebangsaan dan mahasiswa, beliau juga memenangi pingat Emas Sukan SEA empat siri berturut-turut pada 2005, 2007, 2009 dan 2011.\n",
    "\n",
    "Begitu juga atlet kebangsaan seperti Leong Mun Yee (UPM); Pandalela Renong (UM); Bryan Nickson Lomas (UM); Cheng Chu Sian (UPM); Marbawi Sulaiman (UiTM) dan Norasheela Khalid (UPM).\n",
    "\n",
    "Jika disenaraikan, mungkin lebih ramai lagi. Namun, pernah terlintas di fikiran mengapa hanya atlet dari sukan terjun yang dapat memenangi pingat di Sukan Olimpik? Bagaimana dengan atlet lain yang juga layak secara merit? Apakah kekangan atau masalah dihadapi sebagai atlet dan mahasiswa?\n",
    "\n",
    "Adakah kewujudan universiti sukan akan memberi impak besar kepada kemajuan sukan negara? Jika dirancang dan diatur dengan cekap dan sistematik, ia perkara tidak mustahil dicapai.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded = encoder.encode(cleaning(string3)) + [1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8min 57s, sys: 10.1 s, total: 9min 7s\n",
      "Wall time: 49.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "l = sess.run(logits, feed_dict = {X: [encoded], top_p: 0.8, temperature: 0.1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'- Sekiranya anda ingin menjadi atlet Olimpik, anda mungkin mahu mengekalkannya. Itulah hasil kajian baru dari Universiti Kebangsaan Malaysia, yang mendapati bahawa pelajar yang sudah bersara, yang sering mengambil pendekatan yang lebih baik untuk menjadi atlet di Sukan Asia. \"Dengan demikian, pelajar yang sudah bersara itu memberi kesan positif prestasi dan prestasi bukannya memberi kesan yang sama seperti atlet lain,\" kata seorang pakar sukan. \"Lain kali, pelajar yang sudah bersara, mengambil pendekatan yang lebih baik untuk menjadi atlet di Sukan Asia. \" Tetapi ada yang mengatakan bahawa pelajar yang sudah bersara, yang sering mengambil pendekatan yang lebih baik untuk menjadi atlet di Sukan Asia. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Lain kali, pelajar yang sudah bersara, mengambil pendekatan yang lebih baik untuk menjadi atlet di Sukan Asia. \" Tetapi ada yang mengatakan bahawa pelajar yang sudah bersara, yang sering mengambil pendekatan yang lebih baik untuk menjadi atlet di Sukan Asia. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. \"Memandu dan memberi tumpuan kepada kejayaan yang dicapai pada tahun 2007,\" kata seorang pakar sukan. (Inilah sebabnya mengapa atlet tidak boleh menjadi atlet Olimpik.'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.decode(l[0].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'output/model.ckpt'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'output/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'top_p',\n",
       " 'temperature',\n",
       " 'embeddings/weights',\n",
       " 'encoder/layer_0/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_0/attention/self/query/kernel',\n",
       " 'encoder/layer_0/attention/self/key/kernel',\n",
       " 'encoder/layer_0/attention/self/value/kernel',\n",
       " 'encoder/layer_0/attention/self/Softmax',\n",
       " 'encoder/layer_0/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_0/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_0/ffn/dense/kernel',\n",
       " 'encoder/layer_0/ffn/dense/bias',\n",
       " 'encoder/layer_0/ffn/dense_1/kernel',\n",
       " 'encoder/layer_0/ffn/dense_1/bias',\n",
       " 'encoder/layer_1/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_1/attention/self/query/kernel',\n",
       " 'encoder/layer_1/attention/self/key/kernel',\n",
       " 'encoder/layer_1/attention/self/value/kernel',\n",
       " 'encoder/layer_1/attention/self/Softmax',\n",
       " 'encoder/layer_1/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_1/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_1/ffn/dense/kernel',\n",
       " 'encoder/layer_1/ffn/dense/bias',\n",
       " 'encoder/layer_1/ffn/dense_1/kernel',\n",
       " 'encoder/layer_1/ffn/dense_1/bias',\n",
       " 'encoder/layer_2/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_2/attention/self/query/kernel',\n",
       " 'encoder/layer_2/attention/self/key/kernel',\n",
       " 'encoder/layer_2/attention/self/value/kernel',\n",
       " 'encoder/layer_2/attention/self/Softmax',\n",
       " 'encoder/layer_2/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_2/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_2/ffn/dense/kernel',\n",
       " 'encoder/layer_2/ffn/dense/bias',\n",
       " 'encoder/layer_2/ffn/dense_1/kernel',\n",
       " 'encoder/layer_2/ffn/dense_1/bias',\n",
       " 'encoder/layer_3/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_3/attention/self/query/kernel',\n",
       " 'encoder/layer_3/attention/self/key/kernel',\n",
       " 'encoder/layer_3/attention/self/value/kernel',\n",
       " 'encoder/layer_3/attention/self/Softmax',\n",
       " 'encoder/layer_3/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_3/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_3/ffn/dense/kernel',\n",
       " 'encoder/layer_3/ffn/dense/bias',\n",
       " 'encoder/layer_3/ffn/dense_1/kernel',\n",
       " 'encoder/layer_3/ffn/dense_1/bias',\n",
       " 'encoder/layer_4/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_4/attention/self/query/kernel',\n",
       " 'encoder/layer_4/attention/self/key/kernel',\n",
       " 'encoder/layer_4/attention/self/value/kernel',\n",
       " 'encoder/layer_4/attention/self/Softmax',\n",
       " 'encoder/layer_4/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_4/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_4/ffn/dense/kernel',\n",
       " 'encoder/layer_4/ffn/dense/bias',\n",
       " 'encoder/layer_4/ffn/dense_1/kernel',\n",
       " 'encoder/layer_4/ffn/dense_1/bias',\n",
       " 'encoder/layer_5/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_5/attention/self/query/kernel',\n",
       " 'encoder/layer_5/attention/self/key/kernel',\n",
       " 'encoder/layer_5/attention/self/value/kernel',\n",
       " 'encoder/layer_5/attention/self/Softmax',\n",
       " 'encoder/layer_5/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_5/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_5/ffn/dense/kernel',\n",
       " 'encoder/layer_5/ffn/dense/bias',\n",
       " 'encoder/layer_5/ffn/dense_1/kernel',\n",
       " 'encoder/layer_5/ffn/dense_1/bias',\n",
       " 'encoder/layer_6/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_6/attention/self/query/kernel',\n",
       " 'encoder/layer_6/attention/self/key/kernel',\n",
       " 'encoder/layer_6/attention/self/value/kernel',\n",
       " 'encoder/layer_6/attention/self/Softmax',\n",
       " 'encoder/layer_6/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_6/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_6/ffn/dense/kernel',\n",
       " 'encoder/layer_6/ffn/dense/bias',\n",
       " 'encoder/layer_6/ffn/dense_1/kernel',\n",
       " 'encoder/layer_6/ffn/dense_1/bias',\n",
       " 'encoder/layer_7/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_7/attention/self/query/kernel',\n",
       " 'encoder/layer_7/attention/self/key/kernel',\n",
       " 'encoder/layer_7/attention/self/value/kernel',\n",
       " 'encoder/layer_7/attention/self/Softmax',\n",
       " 'encoder/layer_7/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_7/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_7/ffn/dense/kernel',\n",
       " 'encoder/layer_7/ffn/dense/bias',\n",
       " 'encoder/layer_7/ffn/dense_1/kernel',\n",
       " 'encoder/layer_7/ffn/dense_1/bias',\n",
       " 'encoder/layer_8/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_8/attention/self/query/kernel',\n",
       " 'encoder/layer_8/attention/self/key/kernel',\n",
       " 'encoder/layer_8/attention/self/value/kernel',\n",
       " 'encoder/layer_8/attention/self/Softmax',\n",
       " 'encoder/layer_8/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_8/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_8/ffn/dense/kernel',\n",
       " 'encoder/layer_8/ffn/dense/bias',\n",
       " 'encoder/layer_8/ffn/dense_1/kernel',\n",
       " 'encoder/layer_8/ffn/dense_1/bias',\n",
       " 'encoder/layer_9/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_9/attention/self/query/kernel',\n",
       " 'encoder/layer_9/attention/self/key/kernel',\n",
       " 'encoder/layer_9/attention/self/value/kernel',\n",
       " 'encoder/layer_9/attention/self/Softmax',\n",
       " 'encoder/layer_9/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_9/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_9/ffn/dense/kernel',\n",
       " 'encoder/layer_9/ffn/dense/bias',\n",
       " 'encoder/layer_9/ffn/dense_1/kernel',\n",
       " 'encoder/layer_9/ffn/dense_1/bias',\n",
       " 'encoder/layer_10/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_10/attention/self/query/kernel',\n",
       " 'encoder/layer_10/attention/self/key/kernel',\n",
       " 'encoder/layer_10/attention/self/value/kernel',\n",
       " 'encoder/layer_10/attention/self/Softmax',\n",
       " 'encoder/layer_10/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_10/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_10/ffn/dense/kernel',\n",
       " 'encoder/layer_10/ffn/dense/bias',\n",
       " 'encoder/layer_10/ffn/dense_1/kernel',\n",
       " 'encoder/layer_10/ffn/dense_1/bias',\n",
       " 'encoder/layer_11/attention/self/LayerNorm/gamma',\n",
       " 'encoder/layer_11/attention/self/query/kernel',\n",
       " 'encoder/layer_11/attention/self/key/kernel',\n",
       " 'encoder/layer_11/attention/self/value/kernel',\n",
       " 'encoder/layer_11/attention/self/Softmax',\n",
       " 'encoder/layer_11/attention/self/output/dense/kernel',\n",
       " 'encoder/layer_11/ffn/LayerNorm/gamma',\n",
       " 'encoder/layer_11/ffn/dense/kernel',\n",
       " 'encoder/layer_11/ffn/dense/bias',\n",
       " 'encoder/layer_11/ffn/dense_1/kernel',\n",
       " 'encoder/layer_11/ffn/dense_1/bias',\n",
       " 'encoder/LayerNorm/gamma',\n",
       " 'decoder/layer_0/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_0/attention/self/query/kernel',\n",
       " 'decoder/layer_0/attention/self/key/kernel',\n",
       " 'decoder/layer_0/attention/self/value/kernel',\n",
       " 'while/decoder/layer_0/attention/self/Softmax',\n",
       " 'decoder/layer_0/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_0/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_0/memory_attention/query/kernel',\n",
       " 'decoder/layer_0/memory_attention/key/kernel',\n",
       " 'decoder/layer_0/memory_attention/value/kernel',\n",
       " 'decoder/layer_0/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_0/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_0/ffn/dense/kernel',\n",
       " 'decoder/layer_0/ffn/dense/bias',\n",
       " 'decoder/layer_0/ffn/dense_1/kernel',\n",
       " 'decoder/layer_0/ffn/dense_1/bias',\n",
       " 'decoder/layer_1/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_1/attention/self/query/kernel',\n",
       " 'decoder/layer_1/attention/self/key/kernel',\n",
       " 'decoder/layer_1/attention/self/value/kernel',\n",
       " 'while/decoder/layer_1/attention/self/Softmax',\n",
       " 'decoder/layer_1/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_1/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_1/memory_attention/query/kernel',\n",
       " 'decoder/layer_1/memory_attention/key/kernel',\n",
       " 'decoder/layer_1/memory_attention/value/kernel',\n",
       " 'decoder/layer_1/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_1/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_1/ffn/dense/kernel',\n",
       " 'decoder/layer_1/ffn/dense/bias',\n",
       " 'decoder/layer_1/ffn/dense_1/kernel',\n",
       " 'decoder/layer_1/ffn/dense_1/bias',\n",
       " 'decoder/layer_2/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_2/attention/self/query/kernel',\n",
       " 'decoder/layer_2/attention/self/key/kernel',\n",
       " 'decoder/layer_2/attention/self/value/kernel',\n",
       " 'while/decoder/layer_2/attention/self/Softmax',\n",
       " 'decoder/layer_2/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_2/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_2/memory_attention/query/kernel',\n",
       " 'decoder/layer_2/memory_attention/key/kernel',\n",
       " 'decoder/layer_2/memory_attention/value/kernel',\n",
       " 'decoder/layer_2/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_2/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_2/ffn/dense/kernel',\n",
       " 'decoder/layer_2/ffn/dense/bias',\n",
       " 'decoder/layer_2/ffn/dense_1/kernel',\n",
       " 'decoder/layer_2/ffn/dense_1/bias',\n",
       " 'decoder/layer_3/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_3/attention/self/query/kernel',\n",
       " 'decoder/layer_3/attention/self/key/kernel',\n",
       " 'decoder/layer_3/attention/self/value/kernel',\n",
       " 'while/decoder/layer_3/attention/self/Softmax',\n",
       " 'decoder/layer_3/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_3/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_3/memory_attention/query/kernel',\n",
       " 'decoder/layer_3/memory_attention/key/kernel',\n",
       " 'decoder/layer_3/memory_attention/value/kernel',\n",
       " 'decoder/layer_3/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_3/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_3/ffn/dense/kernel',\n",
       " 'decoder/layer_3/ffn/dense/bias',\n",
       " 'decoder/layer_3/ffn/dense_1/kernel',\n",
       " 'decoder/layer_3/ffn/dense_1/bias',\n",
       " 'decoder/layer_4/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_4/attention/self/query/kernel',\n",
       " 'decoder/layer_4/attention/self/key/kernel',\n",
       " 'decoder/layer_4/attention/self/value/kernel',\n",
       " 'while/decoder/layer_4/attention/self/Softmax',\n",
       " 'decoder/layer_4/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_4/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_4/memory_attention/query/kernel',\n",
       " 'decoder/layer_4/memory_attention/key/kernel',\n",
       " 'decoder/layer_4/memory_attention/value/kernel',\n",
       " 'decoder/layer_4/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_4/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_4/ffn/dense/kernel',\n",
       " 'decoder/layer_4/ffn/dense/bias',\n",
       " 'decoder/layer_4/ffn/dense_1/kernel',\n",
       " 'decoder/layer_4/ffn/dense_1/bias',\n",
       " 'decoder/layer_5/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_5/attention/self/query/kernel',\n",
       " 'decoder/layer_5/attention/self/key/kernel',\n",
       " 'decoder/layer_5/attention/self/value/kernel',\n",
       " 'while/decoder/layer_5/attention/self/Softmax',\n",
       " 'decoder/layer_5/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_5/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_5/memory_attention/query/kernel',\n",
       " 'decoder/layer_5/memory_attention/key/kernel',\n",
       " 'decoder/layer_5/memory_attention/value/kernel',\n",
       " 'decoder/layer_5/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_5/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_5/ffn/dense/kernel',\n",
       " 'decoder/layer_5/ffn/dense/bias',\n",
       " 'decoder/layer_5/ffn/dense_1/kernel',\n",
       " 'decoder/layer_5/ffn/dense_1/bias',\n",
       " 'decoder/layer_6/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_6/attention/self/query/kernel',\n",
       " 'decoder/layer_6/attention/self/key/kernel',\n",
       " 'decoder/layer_6/attention/self/value/kernel',\n",
       " 'while/decoder/layer_6/attention/self/Softmax',\n",
       " 'decoder/layer_6/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_6/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_6/memory_attention/query/kernel',\n",
       " 'decoder/layer_6/memory_attention/key/kernel',\n",
       " 'decoder/layer_6/memory_attention/value/kernel',\n",
       " 'decoder/layer_6/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_6/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_6/ffn/dense/kernel',\n",
       " 'decoder/layer_6/ffn/dense/bias',\n",
       " 'decoder/layer_6/ffn/dense_1/kernel',\n",
       " 'decoder/layer_6/ffn/dense_1/bias',\n",
       " 'decoder/layer_7/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_7/attention/self/query/kernel',\n",
       " 'decoder/layer_7/attention/self/key/kernel',\n",
       " 'decoder/layer_7/attention/self/value/kernel',\n",
       " 'while/decoder/layer_7/attention/self/Softmax',\n",
       " 'decoder/layer_7/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_7/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_7/memory_attention/query/kernel',\n",
       " 'decoder/layer_7/memory_attention/key/kernel',\n",
       " 'decoder/layer_7/memory_attention/value/kernel',\n",
       " 'decoder/layer_7/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_7/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_7/ffn/dense/kernel',\n",
       " 'decoder/layer_7/ffn/dense/bias',\n",
       " 'decoder/layer_7/ffn/dense_1/kernel',\n",
       " 'decoder/layer_7/ffn/dense_1/bias',\n",
       " 'decoder/layer_8/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_8/attention/self/query/kernel',\n",
       " 'decoder/layer_8/attention/self/key/kernel',\n",
       " 'decoder/layer_8/attention/self/value/kernel',\n",
       " 'while/decoder/layer_8/attention/self/Softmax',\n",
       " 'decoder/layer_8/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_8/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_8/memory_attention/query/kernel',\n",
       " 'decoder/layer_8/memory_attention/key/kernel',\n",
       " 'decoder/layer_8/memory_attention/value/kernel',\n",
       " 'decoder/layer_8/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_8/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_8/ffn/dense/kernel',\n",
       " 'decoder/layer_8/ffn/dense/bias',\n",
       " 'decoder/layer_8/ffn/dense_1/kernel',\n",
       " 'decoder/layer_8/ffn/dense_1/bias',\n",
       " 'decoder/layer_9/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_9/attention/self/query/kernel',\n",
       " 'decoder/layer_9/attention/self/key/kernel',\n",
       " 'decoder/layer_9/attention/self/value/kernel',\n",
       " 'while/decoder/layer_9/attention/self/Softmax',\n",
       " 'decoder/layer_9/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_9/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_9/memory_attention/query/kernel',\n",
       " 'decoder/layer_9/memory_attention/key/kernel',\n",
       " 'decoder/layer_9/memory_attention/value/kernel',\n",
       " 'decoder/layer_9/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_9/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_9/ffn/dense/kernel',\n",
       " 'decoder/layer_9/ffn/dense/bias',\n",
       " 'decoder/layer_9/ffn/dense_1/kernel',\n",
       " 'decoder/layer_9/ffn/dense_1/bias',\n",
       " 'decoder/layer_10/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_10/attention/self/query/kernel',\n",
       " 'decoder/layer_10/attention/self/key/kernel',\n",
       " 'decoder/layer_10/attention/self/value/kernel',\n",
       " 'while/decoder/layer_10/attention/self/Softmax',\n",
       " 'decoder/layer_10/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_10/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_10/memory_attention/query/kernel',\n",
       " 'decoder/layer_10/memory_attention/key/kernel',\n",
       " 'decoder/layer_10/memory_attention/value/kernel',\n",
       " 'decoder/layer_10/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_10/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_10/ffn/dense/kernel',\n",
       " 'decoder/layer_10/ffn/dense/bias',\n",
       " 'decoder/layer_10/ffn/dense_1/kernel',\n",
       " 'decoder/layer_10/ffn/dense_1/bias',\n",
       " 'decoder/layer_11/attention/self/LayerNorm/gamma',\n",
       " 'decoder/layer_11/attention/self/query/kernel',\n",
       " 'decoder/layer_11/attention/self/key/kernel',\n",
       " 'decoder/layer_11/attention/self/value/kernel',\n",
       " 'while/decoder/layer_11/attention/self/Softmax',\n",
       " 'decoder/layer_11/attention/self/output/dense/kernel',\n",
       " 'decoder/layer_11/memory_attention/LayerNorm/gamma',\n",
       " 'decoder/layer_11/memory_attention/query/kernel',\n",
       " 'decoder/layer_11/memory_attention/key/kernel',\n",
       " 'decoder/layer_11/memory_attention/value/kernel',\n",
       " 'decoder/layer_11/memory_attention/output/dense/kernel',\n",
       " 'decoder/layer_11/ffn/LayerNorm/gamma',\n",
       " 'decoder/layer_11/ffn/dense/kernel',\n",
       " 'decoder/layer_11/ffn/dense/bias',\n",
       " 'decoder/layer_11/ffn/dense_1/kernel',\n",
       " 'decoder/layer_11/ffn/dense_1/bias',\n",
       " 'decoder/LayerNorm/gamma',\n",
       " 'logits']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'top_p' in n.name\n",
    "        or 'temperature' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'gradients' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from output/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-25-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 365 variables.\n",
      "INFO:tensorflow:Converted 365 variables to const ops.\n",
      "10764 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('output', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-29-f9c2d7850f78>:4: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    }
   ],
   "source": [
    "pb = 'output/frozen_model.pb'\n",
    "\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "        \n",
    "inputs = ['Placeholder', 'top_p', 'temperature']\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       inputs,\n",
    "                                       ['logits'], transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename, **kwargs):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "\n",
    "    # https://github.com/onnx/tensorflow-onnx/issues/77#issuecomment-445066091\n",
    "    # to fix import T5\n",
    "    for node in graph_def.node:\n",
    "        if node.op == 'RefSwitch':\n",
    "            node.op = 'Switch'\n",
    "            for index in xrange(len(node.input)):\n",
    "                if 'moving_' in node.input[index]:\n",
    "                    node.input[index] = node.input[index] + '/read'\n",
    "        elif node.op == 'AssignSub':\n",
    "            node.op = 'Sub'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "        elif node.op == 'AssignAdd':\n",
    "            node.op = 'Add'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "        elif node.op == 'Assign':\n",
    "            node.op = 'Identity'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "            if 'validate_shape' in node.attr:\n",
    "                del node.attr['validate_shape']\n",
    "            if len(node.input) == 2:\n",
    "                node.input[0] = node.input[1]\n",
    "                del node.input[1]\n",
    "\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('output/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "top_p = g.get_tensor_by_name('import/top_p:0')\n",
    "temperature = g.get_tensor_by_name('import/temperature:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "test_sess = tf.Session(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4min 42s, sys: 9.96 s, total: 4min 52s\n",
      "Wall time: 43.4 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'- Sekiranya anda ingin menjadi atlet Olimpik, anda mungkin mahu mengekalkannya. Itulah hasil kajian baru dari Universiti Kebangsaan Malaysia, yang mendapati bahawa pelajar yang sudah bersara, yang sering mengambil pendekatan yang lebih baik untuk berjaya seperti Sukan SEA dan Sukan Asia. \"Dengan kecemerlangan dan prestasi yang diberikan, universiti di bawah tekanan dan kurang akademik,\" kata seorang pakar sukan. \"Lain kali, pelajar cemerlang yang memberi prestasi yang kuat dan prestasi yang pantas diberi latihan. \" Dan itu mungkin bermakna pelajar yang sudah bersara tidak akan dapat memperoleh pingat emas dalam tempoh empat tahun lagi. \"Dengan kecemerlangan dan prestasi yang diberikan, universiti di bawah tekanan dan kurang akademik,\" kata seorang pakar sukan. \"Lain kali, pelajar cemerlang yang memberi prestasi yang pantas diberi latihan dan prestasi yang pantas diberi latihan. \" Tetapi ada yang mengatakan bahawa pelajar yang sudah bersara tidak akan dapat memperoleh pingat dalam tempoh empat tahun lagi. \"Memandu jenama yang diberi latihan kemahiran adalah cara yang baik untuk memajukan pasukan kebangsaan,\" kata seorang pakar sukan. \"Lain kali, pelajar cemerlang yang memberi prestasi yang pantas diberi latihan dan prestasi yang pantas diberi latihan. \" (Inilah sebabnya mengapa sukan Olimpik terus berkembang.'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "l = test_sess.run(logits, feed_dict = {x: [encoded], top_p: 0.0, temperature: 0.0})\n",
    "encoder.decode([i for i in l[0].tolist() if i > 0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
